﻿module Tokenize

open System

let tokenize (s : string) =
    // No, tokenize IS NOT split
    let s = s.Trim()
    let tokens = ResizeArray<string>()
    let LENGTH = s.Length
    
    let parseString (pos : int) =
        // 'pos' points to 1st character after opening quote
        if pos >= LENGTH then
            // Be permissive - just in case
            (LENGTH, Some "")
        else
            let endQuote = s.IndexOf( '"', pos )
            let endQuote = if endQuote < 0 then LENGTH else endQuote
            let content = s.Substring( pos, endQuote - pos )
            (endQuote + 1, Some content)

    let parseAtom (pos : int) =
        let startAtom = pos
        let mutable endAtom = pos + 1
        while endAtom < LENGTH && not( Char.IsWhiteSpace( s.[endAtom] ) ) do
            endAtom <- endAtom + 1
        // endAtom points outside of 's' or to a 1st whitespace char after atom
        let content = s.Substring( startAtom, endAtom - startAtom )
        (endAtom, Some content)

    let rec parseNextToken (pos : int) =
        if pos >= LENGTH then
            (LENGTH, None)
        elif s.[pos] = '"' then
            parseString (pos+1)
        elif Char.IsWhiteSpace( s.[pos] ) then
            parseNextToken (pos+1)
        else
            parseAtom pos
    
    let mutable pos = 0
    while pos < LENGTH do
        let pos', optTkn = parseNextToken pos
        pos <- pos'
        match optTkn with
        | Some tkn -> tokens.Add tkn
        | None -> ()

    tokens.ToArray()

